/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package prepocessing_2.com.prep;

import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Text;
import org.jdom.input.SAXBuilder;

/**
 *
 * @author baga
 */
public class Tokenizer {

    private ArrayList docs;
    private ArrayList tokens;
    private final char[] forbChar = {'.', ',', '+', '-', '^', '@', '#', '!',
        '$', '%', '&', '*', '(', ')', '-', '_', '=', '\\', '\'', '/', '\'',
        '\'', '?', '<', '>', ':', ';', '|', '[', ']'};
    private int count = 1;
    private File prepDoc;
    private ArrayList docContent;
    private ArrayList content;

    public Tokenizer() {
        docs = new ArrayList();
        tokens = new ArrayList();
        docContent = new ArrayList();
        content = new ArrayList();
    }

    public void setDocument(File xmlDoc) {
        prepDoc = xmlDoc;
        if (docs.size() > 0) {
            System.out.println("tokenizer size before clearing docs " + docs.size() +
                    " tokens size : " + tokens.size());
            docs.clear();
            tokens.clear();
            System.out.println("tokenizer size after clearing docs " + docs.size() +
                    " tokens size : " + tokens.size());
        }
    }

    public void initialized() {
        if (prepDoc.exists()) {
            SAXBuilder builder = new SAXBuilder();

            try {
                Document doc = builder.build(prepDoc);
                Element root = doc.getRootElement();
                listNodes(root, "");
            } catch (JDOMException je) {
                je.printStackTrace();
            } catch (IOException ie) {
                ie.printStackTrace();
            }
        }
    }

    private void listNodes(Object obj, String path) {
        if (obj instanceof Element) {
            Element children = (Element) obj;
            List listChild = children.getContent();
            Iterator iterate = listChild.iterator();
            while (iterate.hasNext()) {
                Object child = iterate.next();
                listNodes(child, path + "/" + children.getName());
            }
        } else if (obj instanceof Text) {
            Text text = (Text) obj;
            String tokenNormal = text.getTextNormalize();
            if (!tokenNormal.isEmpty()) {
                String[] normalText = SplitNormalizeToken.getNormalTokensStopList(tokenNormal.toLowerCase());
                String constructToken = null;
//                addDocContent(prepDoc.getAbsolutePath() + "#" + count, tokenNormal);
                docContent.add(prepDoc.getAbsolutePath() + "#" + count);
                content.add(tokenNormal);
                for (int i = 0; i < normalText.length; i++) {
                    String norm = normalText[i].trim();
                    if (!norm.equals("") ||
                            !norm.equals(" ") || norm != null || !norm.isEmpty()) {
                        constructToken = path + "#" + norm + "#" + count;
                        if (!tokens.contains(constructToken)) {
                            tokens.add(constructToken);
                            docs.add(prepDoc.getAbsolutePath() + "#" + 1);
                        } else {
                            int idx = tokens.lastIndexOf(constructToken);
                            String tempStr = (String) docs.get(idx);
                            String[] tempSplit = tempStr.split("#");
                            int cnt = Integer.parseInt(tempSplit[1]);
                            cnt++;
                            docs.set(idx, tempSplit[0] + "#" + cnt);
                        }
                    }
                }
                count++;
            }
        }
    }

    public String[] getDocContent() {
        return (String[]) docContent.toArray(new String[docContent.size()]);
    }

    public String[] getContent() {
        return (String[]) content.toArray(new String[content.size()]);
    }

    private void addDocContent(String doc, String content) {
        try {
            String url = "jdbc:mysql://localhost:3306/xsedb";
            Class.forName("com.mysql.jdbc.Driver");
            Connection conn = DriverManager.getConnection(url,
                    "root", "");
//            PreparedStatement ps2 = conn.prepareStatement("select count(*) " +
//                    "from doc_content where doc like ?");
//            ps2.setString(1, "%" + doc + "%");
//            ResultSet rs = ps2.executeQuery();
//            while (rs.next()) {
//                int num = rs.getInt(1);
//                if (num == 0) {
            PreparedStatement ps = conn.prepareStatement("insert into doc_content " +
                    "values(?, ?)");
            ps.setString(1, doc);
            ps.setString(2, content);
            ps.execute();
            ps.clearParameters();
            ps = null;
//                }
//            }
            conn.close();
            conn = null;
        } catch (ClassNotFoundException ce) {
            ce.printStackTrace();
        } catch (SQLException se) {
            se.printStackTrace();
        }
    }

    public String[] getTokens() {
        return (String[]) tokens.toArray(new String[tokens.size()]);
    }

    public String[] getDocs() {
        return (String[]) docs.toArray(new String[docs.size()]);
    }

//    public static void main(String[] args) {
//        Tokenizer tz = new Tokenizer();
//        File useDir = new File("/export/home/baga/Documents/Book/TA/data_xsengine");
//        File[] listFiles = useDir.listFiles();
//        System.out.println(listFiles.length);
//        try {
//            String url = "jdbc:mysql://localhost:3306/xsedb";
//            Class.forName("com.mysql.jdbc.Driver");
//            Connection conn = DriverManager.getConnection(url,
//                    "root", "");
//            PreparedStatement ps = conn.prepareStatement("select doc from doc_content");
//            ResultSet rs = ps.executeQuery();
//            ArrayList abc = new ArrayList();
//            while (rs.next()) {
//                String doc = rs.getString(1);
//                String val = doc.substring(doc.lastIndexOf("/") + 1, doc.indexOf("#"));
//                if (abc.isEmpty() || !abc.contains(val)) {
//                    abc.add(val);
//                }
//            }
//
//            for (File file : listFiles) {
//                // execute All
//                if (!abc.contains(file.getName())) {
//                    tz.setDocument(file);
//                    tz.initialized();
//                }
//            }
//        } catch (SQLException se) {
//        } catch (ClassNotFoundException ce) {
//        }
//    }
}
